package com.me.corpus;

import com.me.dao.CorpusItemMapper;
import com.me.entity.CorpusItem;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Main {
	public  transient  static Log log = LogFactory.getLog(Main.class);
	/**
	 * 表情处理
	 * @throws IOException 
	 * **/
	public static void emotSelect(Map<String, Integer> map) throws IOException {
		
		List<Entry<String,Integer>> list = new ArrayList<Entry<String,Integer>>(map.entrySet());
		FileOutputStream emotOutputStream = new FileOutputStream("emot.txt");
        //然后通过比较器来实现排序
        Collections.sort(list,new Comparator<Entry<String,Integer>>() {
            //升序排序
            public int compare(Entry<String, Integer> o1,
                    Entry<String, Integer> o2) {
                return o2.getValue()-o1.getValue();
            }
            
        });
        
        for(Entry<String,Integer> mapping:list){
               //System.out.println(mapping.getKey()+":"+mapping.getValue());
        	emotOutputStream.write((mapping.getKey()+":"+mapping.getValue()+"\r\n").getBytes());
        }
        emotOutputStream.close();
	}
	public static void main(String[] args) {
		//加载资源文件
		ApplicationContext ctx=null;
		ctx=new ClassPathXmlApplicationContext("applicationContext.xml");
		CorpusItemMapper corpusItemMapper =(CorpusItemMapper)ctx.getBean("corpusItemDao");

		Map<String, Integer> map = new HashMap<String, Integer>();

		//E:\藏语相关\Weibo_zangyu_special
		File root = new File("E:/藏语相关/Weibo_zangyu_special");
		
		if (root.exists()&&root.isDirectory()) {
			
			File[] files = root.listFiles();

			//System.out.println(files.length);
			
			int count = 0;
			for (int i = 0; i < files.length; i++) {
				count++;
				if (count>2000) {
					break;
				}
				try {
					BufferedReader br = new BufferedReader(new FileReader(files[i]));
					String fileName = files[i].getName();
					String w_id = fileName.substring(0, fileName.length()-4);
					//System.out.println(w_id);
					log.debug("微博ID:"+w_id);
					String line = "";

					while ((line=br.readLine())!=null) {
						//System.out.println(line);
						if(line.startsWith("#内容")){
							line = line.substring(4, line.length());
							//实体类
							CorpusItem corpusItem = new CorpusItem();
							/*
							 * 对文本的出才有替换的方式
							 * */
							log.info("微博内容:"+line);
							corpusItem.setItemid(w_id);
							corpusItem.setWeibo(line);
							//去| 阅读全文:
							if (line.contains("| 阅读全文:")) {
								//System.out.println(line); //初始预计
								int start = line.indexOf("| 阅读全文:");
								String tagMsg = line.substring(start, line.length());

								String tag = tagMsg.split(" ")[0];
								String msg = line.replace(tag, "");

								tag = tagMsg.split(" ")[1];
								msg = msg.replace(tag, "");
								line = msg;
							}

							String regex1 = "\\(.*?\\)";
							Pattern pattern = Pattern.compile(regex1);
							Matcher matcher = pattern.matcher(line);
							while(matcher.find()) {
								String tmp = matcher.group();
								log.debug("微博括号内容:"+tmp);
								line = line.replace(tmp, "");
							}

							String regex2 = "（.*?）";
							pattern = Pattern.compile(regex2);
							matcher = pattern.matcher(line);
							while(matcher.find()) {
								String tmp = matcher.group();
								log.debug("微博括号内容:"+tmp);
								line = line.replace(tmp, "");
							}
							//（分享自 @优酷网）

							//我分享了
							if (line.contains("我分享了")) {
								//System.out.println(line); //初始预计
								int start = line.indexOf("我分享了");
								String tagMsg = line.substring(start, line.length());
								String tag = tagMsg.split(" ")[0];//位置信息
								String msg = line.replace(tag, "");
								line = msg;
							}
							if (line.contains("我在这里:")) {
								//System.out.println(line); //初始预计
								int start = line.indexOf("我在这里:");
								String locMsg = line.substring(start, line.length());
								String loc = locMsg.split(" ")[0];//位置信息
								String msg = line.replace(loc, "");
								line = msg;
							}

							//去除藏语微博的地理位置
							while (line.contains("我在:")) {
								//System.out.println(line); //初始预计
								int start = line.indexOf("我在:");
								String locMsg = line.substring(start, line.length());
								String loc = locMsg.split(" ")[0];//位置信息
								String msg = line.replace(loc, "");
								line = msg;
							}
							//去除藏语微博的地理位置
							while (line.contains("[位置]")) {
								//System.out.println(line); //初始预计
								int start = line.indexOf("[位置]");
								String locMsg = line.substring(start, line.length());
								String loc = locMsg.split(" ")[0];//位置信息
								String msg = line.replace(loc, "");
								line = msg;
							}
							//去链接处理
							while (line.contains("http://")) {
								//System.out.println(line);
								int start = line.indexOf("http://");
								String httpMsg = line.substring(start, line.length());
								String http = httpMsg.split(" ")[0];//位置信息
								String msg = line.replace(http, "");
								line = msg;
							}
							//@处理
							while(line.contains("@")) {
								int start = line.indexOf("@");
								String atMsg = line.substring(start, line.length());
								String at = atMsg.split(" ")[0];
								String msg = line.replace(at, "");
								line = msg;
							}
							//匹配[]
							//[\[][\s\S]*[\]]  ^('"\\?)
							//表情处理
							StringBuffer emotmp = new StringBuffer("");
							Matcher emotMatcher = Pattern.compile("\\[[^\\[]*\\]").matcher(line);
							List<String> emotList = new ArrayList<String>();
							while(emotMatcher.find()) {
								String tmp = emotMatcher.group();
								emotList.add(tmp);
								emotmp.append(tmp+" ");//方便训练
								Integer integer = map.get(tmp);
								if(integer==null){
									map.put(tmp, 1);
								}else {
									map.put(tmp, integer+1);
								}
					        }
							for (String string : emotList) {
								line = line.replace(string,"");
							}
							String emot = "";
							if (emotmp.length()>0) {
								emot = emotmp.substring(0, emotmp.length()-1);
								//System.out.println("#"+emot+"#");
								log.debug("微博表情:"+emot);
								corpusItem.setAuxEmot(emot);
							}
							//主题处理
							Matcher topicMatcher = Pattern.compile("#[^\\#]*#").matcher(line);
							List<String> topicList = new ArrayList<String>();
							while(topicMatcher.find()) {
								String tmp = topicMatcher.group();
								topicList.add(tmp);
								log.debug("微博主题:"+tmp);
					        }
							for (String string : topicList) {
								corpusItem.setTopic(string);
								line = line.replace(string,"");
							}
							//处理文本内容

							//获取藏文

							//获取中文信息
							String reg = "[a-zA-Z0-9\u4e00-\u9fa5]+";
							Matcher hanMatcher = Pattern.compile(reg).matcher(line);
							StringBuffer sb = new StringBuffer();
							while(hanMatcher.find()) {
								String tmp = hanMatcher.group();
								line = line.replaceAll(tmp," ");
								sb.append(tmp+"&");
							}
							if (!sb.toString().equals("")){
								String han = sb.toString();
								han = han.substring(0,han.length()-1);
								log.info("微博辅助中文:"+han);
								corpusItem.setHan(han);
							}

							//中文符号处理
							//，。、.!！【 】』『 —-_《》“”"''：>?～
							//log.info("处理后:"+line);
							line = line.replaceAll("[-，。、,•.!！【】『』—_《》“”＂\"''：>?？～‘’\uD83C\uDF32]+","");
							log.info("处理后:"+line.trim());
							if (!line.trim().equals("")){
								corpusItem.setZang(line.trim());

								log.info("Item:"+corpusItem);
							}
							//emotSelect(map);//表情处理


						}
					}
				} catch (FileNotFoundException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
			System.out.println(count);
		}
		
	}
}
